package com.nlp.visualization.crawler.processor;

import com.nlp.visualization.common.CONSTANTS;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * @author LXM
 * @Title: CommonProcessor
 * @Description: 通用的内容爬虫
 * @date 2018/3/13下午7:49
 */
public class CommonProcessor implements PageProcessor {

    Logger logger = LoggerFactory.getLogger(CommonProcessor.class);
    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(CONSTANTS.CRAWLER_RETRY_TIME).setSleepTime(CONSTANTS.CRAWLER_RETRY_WAITING_TIME);


    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    @Override
    public void process(Page page) {

        logger.info("通用内容爬虫-开始爬取：" + page.getUrl());
        //抽取标题
        page.putField("title", page.getHtml().$("title").toString());
        //抽取正文
        page.putField("content", page.getHtml().smartContent().toString());
        //标记URL
        page.putField("url", page.getUrl().toString());

    }

    @Override
    public Site getSite() {
        return site;
    }
}
